{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class node_generator():\n",
    "    '''\n",
    "    产生增强结点\n",
    "    '''\n",
    "    def __init__(self, whiten):\n",
    "        # w矩阵\n",
    "        self.wlist = []\n",
    "        # 偏置向量\n",
    "        self.blist = []\n",
    "        # 非线性函数集合\n",
    "        self.nonlinear = 0\n",
    "    \n",
    "def sigmoid(self,data):\n",
    "    return 1.0/(1+np.exp(-data))\n",
    "\n",
    "def linear(self,data):\n",
    "    return data\n",
    "\n",
    "def tanh(self,data):\n",
    "    return (np.exp(data)-np.exp(-data))/(np.exp(data)+np.exp(-data))\n",
    "\n",
    "def relu(self,data):\n",
    "    return np.maximum(data,0)\n",
    "\n",
    "# 正交投影 施密特正交化\n",
    "def orth(self,W):\n",
    "    for i in range(0,W.shape[1]):\n",
    "        w = np.mat(W[:,i].copy()).T\n",
    "        w_sum = 0\n",
    "        for j in range(i):\n",
    "            wj = np.mat(W[:,j].copy()).T\n",
    "            w_sum += (w.T.dot(wj))[0,0]*wj \n",
    "        w -= w_sum\n",
    "        w = w/np.sqrt(w.T.dot(w))\n",
    "        W[:,i] = np.ravel(w)\n",
    "    return W \n",
    "\n",
    "def generator(self,shape,times):\n",
    "    '''\n",
    "    Parameter: \n",
    "        shape: random shape of w\n",
    "        times: the number of enchanment nodes\n",
    "    '''\n",
    "    for i in range(times):\n",
    "        W = 2*random.random(size=shape)-1\n",
    "        if self.whiten == True:\n",
    "            W = self.orth(W)\n",
    "        b = 2*random.random()-1\n",
    "        # 产生generator\n",
    "        yield (W,b)\n",
    "\n",
    "def generator_nodes(self, data, times, batchsize, nonlinear):\n",
    "    '''\n",
    "    Parameters:\n",
    "        data: picture matrix\n",
    "        times: the number of nodes\n",
    "        batchsize: \n",
    "        nonliner: the dictionary of nonlinear activative function\n",
    "    '''\n",
    "    self.Wlist = [elem[0] for elem in self.generator((data.shape[1],batchsize),times)]\n",
    "    self.blist = [elem[1] for elem in self.generator((data.shape[1],batchsize),times)]\n",
    "    \n",
    "    self.nonlinear = {'linear':self.linear,\n",
    "                        'sigmoid':self.sigmoid,\n",
    "                        'tanh':self.tanh,\n",
    "                        'relu':self.relu\n",
    "                        }[nonlinear]\n",
    "    nodes = self.nonlinear(data.dot(self.Wlist[0])+self.blist[0])\n",
    "    for i in range(1,len(self.Wlist)):\n",
    "        nodes = np.column_stack((nodes, self.nonlinear(data.dot(self.Wlist[i])+self.blist[i])))\n",
    "    return nodes\n",
    "\n",
    "def transform(self,testdata):\n",
    "    # 将测试data映射成所有feature mapping，相当于提取特征\n",
    "    testnodes = self.nonlinear(testdata.dot(self.Wlist[0])+self.blist[0])\n",
    "    for i in range(1,len(self.Wlist)):\n",
    "        testnodes = np.column_stack((testnodes, self.nonlinear(testdata.dot(self.Wlist[i])+self.blist[i])))\n",
    "    return testnodes \n",
    "\n",
    "def update(self,otherW, otherb):\n",
    "    # 参数更新\n",
    "    self.Wlist += otherW\n",
    "    self.blist += otherb \n",
    "\n",
    "class scaler():\n",
    "    '''\n",
    "    归一化操作\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        self._mean = 0\n",
    "        self._std = 0\n",
    "    \n",
    "    def fit_transform(self,traindata):\n",
    "        self._mean = traindata.mean(axis = 0)\n",
    "        self._std = traindata.std(axis = 0)\n",
    "        return (traindata-self._mean)/(self._std+0.001)\n",
    "    \n",
    "    def transform(self,testdata):\n",
    "        return (testdata-self._mean)/(self._std+0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class broadnet:\n",
    "    def __init__(self, \n",
    "                 maptimes = 10, \n",
    "                 enhencetimes = 10,\n",
    "                 map_function = 'linear',\n",
    "                 enhence_function = 'linear',\n",
    "                 batchsize = 'auto', \n",
    "                 reg = 0.001):\n",
    "        \n",
    "        self._maptimes = maptimes\n",
    "        self._enhencetimes = enhencetimes\n",
    "        self._batchsize = batchsize\n",
    "        self._reg = reg\n",
    "        self._map_function = map_function\n",
    "        self._enhence_function = enhence_function\n",
    "        \n",
    "        self.W = 0\n",
    "        self.pesuedoinverse = 0\n",
    "        self.normalscaler = scaler()\n",
    "        self.onehotencoder = preprocessing.OneHotEncoder(sparse = False)\n",
    "        self.mapping_generator = node_generator()\n",
    "        self.enhence_generator = node_generator(whiten = True)\n",
    "\n",
    "    def fit(self,data,label):\n",
    "        if self._batchsize == 'auto':\n",
    "            self._batchsize = data.shape[1]\n",
    "        data = self.normalscaler.fit_transform(data)\n",
    "        label = self.onehotencoder.fit_transform(np.mat(label).T)\n",
    "        \n",
    "        mappingdata = self.mapping_generator.generator_nodes(data,self._maptimes,self._batchsize,self._map_function)\n",
    "        enhencedata = self.enhence_generator.generator_nodes(mappingdata,self._enhencetimes,self._batchsize,self._enhence_function)\n",
    "        \n",
    "        print('number of mapping nodes {0}, number of enhence nodes {1}'.format(mappingdata.shape[1],enhencedata.shape[1]))\n",
    "        print('mapping nodes maxvalue {0} minvalue {1} '.format(round(np.max(mappingdata),5),round(np.min(mappingdata),5)))\n",
    "        print('enhence nodes maxvalue {0} minvalue {1} '.format(round(np.max(enhencedata),5),round(np.min(enhencedata),5)))\n",
    "        \n",
    "        inputdata = np.column_stack((mappingdata,enhencedata))\n",
    "        pesuedoinverse = self.pinv(inputdata,self._reg)\n",
    "        self.W =  pesuedoinverse.dot(label)\n",
    "\n",
    "    # 计算伪逆        \n",
    "    def pinv(self,A,reg):\n",
    "        return np.mat(reg*np.eye(A.shape[1])+A.T.dot(A)).I.dot(A.T)\n",
    "    \n",
    "    def decode(self,Y_onehot):\n",
    "        Y = []\n",
    "        for i in range(Y_onehot.shape[0]):\n",
    "            lis = np.ravel(Y_onehot[i,:]).tolist()\n",
    "            Y.append(lis.index(max(lis)))\n",
    "        return np.array(Y)\n",
    "    \n",
    "    def accuracy(self,predictlabel,label):\n",
    "        # 让多维数组变成一维\n",
    "        label = np.ravel(label).tolist()\n",
    "        predictlabel = predictlabel.tolist()\n",
    "        count = 0\n",
    "        for i in range(len(label)):\n",
    "            if label[i] == predictlabel[i]:\n",
    "                count += 1\n",
    "        return (round(count/len(label),5))\n",
    "        \n",
    "    def predict(self,testdata):\n",
    "        testdata = self.normalscaler.transform(testdata)\n",
    "        test_mappingdata = self.mapping_generator.transform(testdata)\n",
    "        test_enhencedata = self.enhence_generator.transform(test_mappingdata)\n",
    "        \n",
    "        test_inputdata = np.column_stack((test_mappingdata,test_enhencedata))    \n",
    "        return self.decode(test_inputdata.dot(self.W))      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os.path import join\n",
    "import struct\n",
    "from array import array\n",
    "class MnistDataloader(object):\n",
    "    def __init__(self, training_images_filepath,training_labels_filepath,\n",
    "                 test_images_filepath, test_labels_filepath):\n",
    "        self.training_images_filepath = training_images_filepath\n",
    "        self.training_labels_filepath = training_labels_filepath\n",
    "        self.test_images_filepath = test_images_filepath\n",
    "        self.test_labels_filepath = test_labels_filepath\n",
    "    \n",
    "    def read_images_labels(self, images_filepath, labels_filepath):        \n",
    "        labels = []\n",
    "        with open(labels_filepath, 'rb') as file:\n",
    "            magic, size = struct.unpack(\">II\", file.read(8))\n",
    "            if magic != 2049:\n",
    "                raise ValueError('Magic number mismatch, expected 2049, got {}'.format(magic))\n",
    "            labels = array(\"B\", file.read())        \n",
    "        \n",
    "        with open(images_filepath, 'rb') as file:\n",
    "            magic, size, rows, cols = struct.unpack(\">IIII\", file.read(16))\n",
    "            if magic != 2051:\n",
    "                raise ValueError('Magic number mismatch, expected 2051, got {}'.format(magic))\n",
    "            image_data = array(\"B\", file.read())        \n",
    "        images = []\n",
    "        for i in range(size):\n",
    "            images.append([0] * rows * cols)\n",
    "        for i in range(size):\n",
    "            img = np.array(image_data[i * rows * cols:(i + 1) * rows * cols])\n",
    "            img = img.reshape(28, 28)\n",
    "            images[i][:] = np.mat(img)            \n",
    "        \n",
    "        return images, labels\n",
    "            \n",
    "    def load_data(self):\n",
    "        x_train, y_train = self.read_images_labels(self.training_images_filepath, self.training_labels_filepath)\n",
    "        x_test, y_test = self.read_images_labels(self.test_images_filepath, self.test_labels_filepath)\n",
    "        return (x_train, y_train),(x_test, y_test)        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "matrix must be 2-dimensional",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32md:\\project\\code_study\\BLS.ipynb Cell 8'\u001b[0m in \u001b[0;36m<cell line: 38>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=33'>34</a>\u001b[0m \u001b[39m#\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=34'>35</a>\u001b[0m \u001b[39m# Load MINST dataset\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=35'>36</a>\u001b[0m \u001b[39m#\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=36'>37</a>\u001b[0m mnist_dataloader \u001b[39m=\u001b[39m MnistDataloader(training_images_filepath, training_labels_filepath, test_images_filepath, test_labels_filepath)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=37'>38</a>\u001b[0m x_train, y_train, x_test, y_test \u001b[39m=\u001b[39m mnist_dataloader\u001b[39m.\u001b[39;49mload_data()\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000010?line=38'>39</a>\u001b[0m \u001b[39mprint\u001b[39m(x_train[\u001b[39m0\u001b[39m])\n",
      "\u001b[1;32md:\\project\\code_study\\BLS.ipynb Cell 7'\u001b[0m in \u001b[0;36mMnistDataloader.load_data\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=34'>35</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mload_data\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=35'>36</a>\u001b[0m     x_train, y_train \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mread_images_labels(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining_images_filepath, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining_labels_filepath)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=36'>37</a>\u001b[0m     x_test, y_test \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mread_images_labels(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtest_images_filepath, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtest_labels_filepath)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=37'>38</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m (x_train, y_train),(x_test, y_test)\n",
      "\u001b[1;32md:\\project\\code_study\\BLS.ipynb Cell 7'\u001b[0m in \u001b[0;36mMnistDataloader.read_images_labels\u001b[1;34m(self, images_filepath, labels_filepath)\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=29'>30</a>\u001b[0m     img \u001b[39m=\u001b[39m img\u001b[39m.\u001b[39mreshape(\u001b[39m28\u001b[39m, \u001b[39m28\u001b[39m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=30'>31</a>\u001b[0m     images[i][:] \u001b[39m=\u001b[39m img            \n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/project/code_study/BLS.ipynb#ch0000011?line=32'>33</a>\u001b[0m \u001b[39mreturn\u001b[39;00m np\u001b[39m.\u001b[39;49mmat(images), np\u001b[39m.\u001b[39mravel(labels)\n",
      "File \u001b[1;32mD:\\program\\miniconda3\\envs\\bls\\lib\\site-packages\\numpy\\matrixlib\\defmatrix.py:69\u001b[0m, in \u001b[0;36masmatrix\u001b[1;34m(data, dtype)\u001b[0m\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=35'>36</a>\u001b[0m \u001b[39m@set_module\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mnumpy\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=36'>37</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39masmatrix\u001b[39m(data, dtype\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=37'>38</a>\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=38'>39</a>\u001b[0m \u001b[39m    Interpret the input as a matrix.\u001b[39;00m\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=39'>40</a>\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=66'>67</a>\u001b[0m \n\u001b[0;32m     <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=67'>68</a>\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m---> <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=68'>69</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m matrix(data, dtype\u001b[39m=\u001b[39;49mdtype, copy\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n",
      "File \u001b[1;32mD:\\program\\miniconda3\\envs\\bls\\lib\\site-packages\\numpy\\matrixlib\\defmatrix.py:149\u001b[0m, in \u001b[0;36mmatrix.__new__\u001b[1;34m(subtype, data, dtype, copy)\u001b[0m\n\u001b[0;32m    <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=146'>147</a>\u001b[0m shape \u001b[39m=\u001b[39m arr\u001b[39m.\u001b[39mshape\n\u001b[0;32m    <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=147'>148</a>\u001b[0m \u001b[39mif\u001b[39;00m (ndim \u001b[39m>\u001b[39m \u001b[39m2\u001b[39m):\n\u001b[1;32m--> <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=148'>149</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mmatrix must be 2-dimensional\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=149'>150</a>\u001b[0m \u001b[39melif\u001b[39;00m ndim \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m    <a href='file:///d%3A/program/miniconda3/envs/bls/lib/site-packages/numpy/matrixlib/defmatrix.py?line=150'>151</a>\u001b[0m     shape \u001b[39m=\u001b[39m (\u001b[39m1\u001b[39m, \u001b[39m1\u001b[39m)\n",
      "\u001b[1;31mValueError\u001b[0m: matrix must be 2-dimensional"
     ]
    }
   ],
   "source": [
    "#\n",
    "# Verify Reading Dataset via MnistDataloader class\n",
    "#\n",
    "%matplotlib inline\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#\n",
    "# Set file paths based on added MNIST Datasets\n",
    "#\n",
    "input_path = 'dataset\\\\Minist'\n",
    "training_images_filepath = join(input_path, 'train-images-idx3-ubyte\\\\train-images-idx3-ubyte')\n",
    "training_labels_filepath = join(input_path, 'train-labels-idx1-ubyte\\\\train-labels-idx1-ubyte')\n",
    "test_images_filepath = join(input_path, 't10k-images-idx3-ubyte\\\\t10k-images-idx3-ubyte')\n",
    "test_labels_filepath = join(input_path, 't10k-labels-idx1-ubyte\\\\t10k-labels-idx1-ubyte')\n",
    "\n",
    "#\n",
    "# Helper function to show a list of images with their relating titles\n",
    "#\n",
    "def show_images(images, title_texts):\n",
    "    cols = 5\n",
    "    rows = int(len(images)/cols) + 1\n",
    "    plt.figure(figsize=(30,20))\n",
    "    index = 1    \n",
    "    for x in zip(images, title_texts):        \n",
    "        image = x[0]        \n",
    "        title_text = x[1]\n",
    "        plt.subplot(rows, cols, index)        \n",
    "        plt.imshow(image, cmap=plt.cm.gray)\n",
    "        if (title_text != ''):\n",
    "            plt.title(title_text, fontsize = 15);        \n",
    "        index += 1\n",
    "\n",
    "#\n",
    "# Load MINST dataset\n",
    "#\n",
    "mnist_dataloader = MnistDataloader(training_images_filepath, training_labels_filepath, test_images_filepath, test_labels_filepath)\n",
    "x_train, y_train, x_test, y_test = mnist_dataloader.load_data()\n",
    "print(x_train[0])\n",
    "\n",
    "#\n",
    "# Show some random training and test images \n",
    "#\n",
    "# images_2_show = []\n",
    "# titles_2_show = []\n",
    "# for i in range(0, 10):\n",
    "#     r = random.randint(1, 60000)\n",
    "#     images_2_show.append(x_train[r])\n",
    "#     titles_2_show.append('training image [' + str(r) + '] = ' + str(y_train[r]))    \n",
    "\n",
    "# for i in range(0, 5):\n",
    "#     r = random.randint(1, 10000)\n",
    "#     images_2_show.append(x_test[r])        \n",
    "#     titles_2_show.append('test image [' + str(r) + '] = ' + str(y_test[r]))    \n",
    "\n",
    "# show_images(images_2_show, titles_2_show)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 3, 4, 5])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "a = np.array([1, 2, 3, 4])\n",
    "b = np.array([1])\n",
    "a+b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "46e47337159172b8798981ad5f45fb651ac32f92dc4ae0869149af89994345a6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.13 ('bls')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
